﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace InformationRetrieval.Tokenizer
{
    /// <summary>
    /// Provides methods for splitting input text into tokens
    /// </summary>
    public interface ITokenizer
    {
        /// <summary>
        /// Takes a TextReader instance and splits its contents into tokens.
        /// if 'filterStopwords' is true, stopwords are filtered out.
        /// </summary>
        /// <param name="reader">A TextReader instance; must be opened and ready to read</param>
        /// <returns>A (possibly empty) sequence of tokens and their positions</returns>
        IEnumerable<PositionedToken> Tokenize(TextReader reader, bool filterStopwords);
    }

    /// <summary>
    /// Encapsulates one token and its position in its source stream
    /// </summary>
    public struct PositionedToken
    {
        public string Token;
        public int StreamPosition;

        public PositionedToken(string token, int position)
        {
            Token = token;
            StreamPosition = position;
        }
    }
}
